From c70b3e5890b00e3dff2407cc50aafd07ec17b6b3 Mon Sep 17 00:00:00 2001 From: "kfraser@localhost.localdomain" Date: Wed, 28 Feb 2007 14:13:09 +0000 Subject: [PATCH] More PV save/restore fixes. Related changesets: 14148:b67c253d1cdb4f502dec2 13519:b4a8000e76db6b4b27341 These three changesets must be applied as a set! Signed-off-by: Keir Fraser --- .../drivers/xen/xenbus/xenbus_comms.c | 23 +++++++++-- .../drivers/xen/xenbus/xenbus_comms.h | 1 + .../drivers/xen/xenbus/xenbus_xs.c | 39 +++++++++++++------ tools/python/xen/xend/XendCheckpoint.py | 10 ++--- tools/xenstore/xenstored_domain.c | 13 +++++-- 5 files changed, 61 insertions(+), 25 deletions(-) diff --git a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.c b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.c index cee8d74e8f..c00aaf7f6f 100644 --- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.c +++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.c @@ -137,12 +137,15 @@ int xb_write(const void *data, unsigned len) return 0; } -int xb_wait_for_data_to_read(void) +int xb_data_to_read(void) { struct xenstore_domain_interface *intf = xen_store_interface; - return wait_event_interruptible( - xb_waitq, - intf->rsp_cons != intf->rsp_prod); + return (intf->rsp_cons != intf->rsp_prod); +} + +int xb_wait_for_data_to_read(void) +{ + return wait_event_interruptible(xb_waitq, xb_data_to_read()); } int xb_read(void *data, unsigned len) @@ -197,8 +200,20 @@ int xb_read(void *data, unsigned len) /* Set up interrupt handler off store event channel. */ int xb_init_comms(void) { + struct xenstore_domain_interface *intf = xen_store_interface; int err; + if (intf->req_prod != intf->req_cons) + printk(KERN_ERR "XENBUS request ring is not quiescent " + "(%08x:%08x)!\n", intf->req_cons, intf->req_prod); + + if (intf->rsp_prod != intf->rsp_cons) { + printk(KERN_WARNING "XENBUS response ring is not quiescent " + "(%08x:%08x): fixing up\n", + intf->rsp_cons, intf->rsp_prod); + intf->rsp_cons = intf->rsp_prod; + } + if (xenbus_irq) unbind_from_irqhandler(xenbus_irq, &xb_waitq); diff --git a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.h b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.h index 95f10e1255..505100dace 100644 --- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.h +++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_comms.h @@ -37,6 +37,7 @@ int xb_init_comms(void); /* Low level routines. */ int xb_write(const void *data, unsigned len); int xb_read(void *data, unsigned len); +int xb_data_to_read(void); int xb_wait_for_data_to_read(void); int xs_input_avail(void); extern struct xenstore_domain_interface *xen_store_interface; diff --git a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_xs.c b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_xs.c index 69a941d509..5e88ac6f08 100644 --- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_xs.c +++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_xs.c @@ -80,6 +80,9 @@ struct xs_handle { /* One request at a time. */ struct mutex request_mutex; + /* Protect xenbus reader thread against save/restore. */ + struct mutex response_mutex; + /* Protect transactions against save/restore. */ struct rw_semaphore suspend_mutex; }; @@ -654,6 +657,7 @@ void xs_suspend(void) { down_write(&xs_state.suspend_mutex); mutex_lock(&xs_state.request_mutex); + mutex_lock(&xs_state.response_mutex); } void xs_resume(void) @@ -661,6 +665,7 @@ void xs_resume(void) struct xenbus_watch *watch; char token[sizeof(watch) * 2 + 1]; + mutex_unlock(&xs_state.response_mutex); mutex_unlock(&xs_state.request_mutex); /* No need for watches_lock: the suspend_mutex is sufficient. */ @@ -674,6 +679,7 @@ void xs_resume(void) void xs_suspend_cancel(void) { + mutex_unlock(&xs_state.response_mutex); mutex_unlock(&xs_state.request_mutex); up_write(&xs_state.suspend_mutex); } @@ -737,19 +743,27 @@ static int process_msg(void) char *body; int err; - err = xb_wait_for_data_to_read(); - if (err) - return err; - - msg = kmalloc(sizeof(*msg), GFP_KERNEL); - if (msg == NULL) - return -ENOMEM; - /* - * We are now committed to reading an entire message. Partial reads - * across save/restore leave us out of sync with the xenstore daemon. + * We must disallow save/restore while reading a xenstore message. + * A partial read across s/r leaves us out of sync with xenstored. */ - down_read(&xs_state.suspend_mutex); + for (;;) { + err = xb_wait_for_data_to_read(); + if (err) + return err; + mutex_lock(&xs_state.response_mutex); + if (xb_data_to_read()) + break; + /* We raced with save/restore: pending data 'disappeared'. */ + mutex_unlock(&xs_state.response_mutex); + } + + + msg = kmalloc(sizeof(*msg), GFP_KERNEL); + if (msg == NULL) { + err = -ENOMEM; + goto out; + } err = xb_read(&msg->hdr, sizeof(msg->hdr)); if (err) { @@ -803,7 +817,7 @@ static int process_msg(void) } out: - up_read(&xs_state.suspend_mutex); + mutex_unlock(&xs_state.response_mutex); return err; } @@ -833,6 +847,7 @@ int xs_init(void) init_waitqueue_head(&xs_state.reply_waitq); mutex_init(&xs_state.request_mutex); + mutex_init(&xs_state.response_mutex); init_rwsem(&xs_state.suspend_mutex); /* Initialize the shared memory rings to talk to xenstored */ diff --git a/tools/python/xen/xend/XendCheckpoint.py b/tools/python/xen/xend/XendCheckpoint.py index ea01ffd8e9..e94d9fbde7 100644 --- a/tools/python/xen/xend/XendCheckpoint.py +++ b/tools/python/xen/xend/XendCheckpoint.py @@ -230,11 +230,7 @@ def restore(xd, fd, dominfo = None, paused = False): if not is_hvm and handler.console_mfn is None: raise XendError('Could not read console MFN') - dominfo.waitForDevices() # Wait for backends to set up - if not paused: - dominfo.unpause() - - # get qemu state and create a tmp file for dm restore + # get qemu state and create a tmp file for dm restore if is_hvm: qemu_signature = read_exact(fd, len(QEMU_SIGNATURE), "invalid device model signature read") @@ -257,6 +253,10 @@ def restore(xd, fd, dominfo = None, paused = False): dominfo.completeRestore(handler.store_mfn, handler.console_mfn) + dominfo.waitForDevices() # Wait for backends to set up + if not paused: + dominfo.unpause() + return dominfo except: dominfo.destroy() diff --git a/tools/xenstore/xenstored_domain.c b/tools/xenstore/xenstored_domain.c index 68f0816230..ba3507fdfc 100644 --- a/tools/xenstore/xenstored_domain.c +++ b/tools/xenstore/xenstored_domain.c @@ -298,6 +298,7 @@ void do_introduce(struct connection *conn, struct buffered_data *in) unsigned int domid; unsigned long mfn; evtchn_port_t port; + int rc; if (get_strings(in, vec, ARRAY_SIZE(vec)) < ARRAY_SIZE(vec)) { send_error(conn, EINVAL); @@ -341,18 +342,22 @@ void do_introduce(struct connection *conn, struct buffered_data *in) talloc_steal(domain->conn, domain); fire_watches(conn, "@introduceDomain", false); - } - else { - int rc; - + } else if (domain->mfn == mfn) { /* Use XS_INTRODUCE for recreating the xenbus event-channel. */ if (domain->port) xc_evtchn_unbind(xce_handle, domain->port); rc = xc_evtchn_bind_interdomain(xce_handle, domid, port); domain->port = (rc == -1) ? 0 : rc; domain->remote_port = port; + } else { + send_error(conn, EINVAL); + return; } + /* Rings must be quiesced. */ + domain->interface->req_cons = domain->interface->req_prod = 0; + domain->interface->rsp_cons = domain->interface->rsp_prod = 0; + send_ack(conn, XS_INTRODUCE); } -- 2.30.2